- name: d8.prometheus-madison-integration.sending-alerts
  rules:
    - alert: D8PrometheusMadisonErrorSendingAlertsToBackend
      for: 5m
      expr: |
        max by (pod, madison_backend) (
          max by (pod_ip) (
              label_replace(
                rate(prometheus_notifications_errors_total[5m]) / (rate(prometheus_notifications_sent_total[5m])) * 100 > 0,
                "pod_ip", "$1", "alertmanager", ".*://(.*):.*")
          )
          * on (pod_ip) group_right()
          max by (madison_backend, pod, pod_ip) (
            max by (pod, pod_ip) (kube_pod_info)
            * on (pod) group_left(madison_backend)
            label_replace(
              kube_pod_labels{namespace="d8-monitoring", label_madison_backend!=""},
            "madison_backend", "$1", "label_madison_backend", "(.*)")
          )
        )
        * on (pod) group_left(deployment)
        (
          label_replace(kube_controller_pod{namespace="d8-monitoring"}, "deployment", "$1", "controller_name", "(.*)")
        )
      labels:
        severity_level: "{{ if lt $value 0.2 }}9{{ else if lt $value 0.7 }}8{{ else }}7{{ end }}"
        tier: cluster
        d8_module: flant-integration
        d8_component: madison-proxy
      annotations:
        plk_enable_event_severity_change: "true"
        plk_markup_format: "markdown"
        plk_protocol_version: "1"
        plk_labels_as_annotations: "pod"
        plk_create_group_if_not_exists__d8_prometheus_madison_integration_malfunctioning: "D8PrometheusMadisonIntegrationMalfunctioning,tier=cluster,prometheus=deckhouse,namespace=d8-monitoring,deployment={{ $labels.deployment }},kubernetes=~kubernetes"
        plk_grouped_by__d8_prometheus_madison_integration_malfunctioning: "D8PrometheusMadisonIntegrationMalfunctioning,tier=cluster,prometheus=deckhouse,namespace=d8-monitoring,deployment={{ $labels.deployment }},kubernetes=~kubernetes"
        description: |
          Prometheus is unable to deliver {{ $value | humanizePercentage }} alerts to the {{ $labels.madison_backend }} Madison backend using the {{ $labels.pod }} `madison-proxy`.

          You need to check the `madison-proxy` logs: `kubectl -n d8-monitoring logs -f {{ $labels.pod }}`
        summary: Prometheus is unable to deliver {{ $value | humanizePercentage }} alerts to the {{ $labels.madison_backend }} Madison backend using the {{ $labels.pod }} `madison-proxy`.

    - alert: D8PrometheusMadisonErrorSendingAlerts
      for: 10m
      expr: |
        count(
          max by (pod, madison_backend) (
            max by (pod_ip) (
                label_replace(
                  max by (pod, alertmanager) (
                    1 - (increase(prometheus_notifications_successfully_sent_total{namespace="d8-monitoring"}[__SCRAPE_INTERVAL_X_4__]) / increase(prometheus_notifications_sent_total{namespace="d8-monitoring"}[__SCRAPE_INTERVAL_X_4__])) == 1),
                  "pod_ip", "$1", "alertmanager", ".*://(.*):.*")
            )
            * on (pod_ip) group_right()
            max by (madison_backend, pod, pod_ip) (
              max by (pod, pod_ip) (kube_pod_info)
              * on (pod) group_left(madison_backend)
              label_replace(
                kube_pod_labels{namespace="d8-monitoring", label_madison_backend!=""},
              "madison_backend", "$1", "label_madison_backend", "(.*)")
            )
          )
          * on (pod) group_left(deployment)
          (
            label_replace(kube_controller_pod{namespace="d8-monitoring"}, "deployment", "$1", "controller_name", "(.*)")
          )
        ) > 1
      labels:
        severity_level: "6"
        tier: cluster
        d8_module: flant-integration
        d8_component: madison-proxy
      annotations:
        plk_markup_format: "markdown"
        plk_protocol_version: "1"
        plk_create_group_if_not_exists__d8_prometheus_madison_integration_malfunctioning: "D8PrometheusMadisonIntegrationMalfunctioning,tier=cluster,prometheus=deckhouse,namespace=d8-monitoring,deployment={{ $labels.deployment }},kubernetes=~kubernetes"
        plk_grouped_by__d8_prometheus_madison_integration_malfunctioning: "D8PrometheusMadisonIntegrationMalfunctioning,tier=cluster,prometheus=deckhouse,namespace=d8-monitoring,deployment={{ $labels.deployment }},kubernetes=~kubernetes"
        description: |
          Prometheus is unable to deliver 100% alerts through one or more madison-proxies.

          You need to check the `madison-proxy` logs: `kubectl -n d8-monitoring logs -f -l app=madison-proxy`
        summary: Prometheus is unable to deliver 100% alerts through one or more madison-proxies.

    - alert: D8PrometheusMadisonErrorSendingAlerts
      expr: |
        count(
          max by (pod, madison_backend) (
            max by (pod_ip) (
                label_replace(
                  max by (pod, alertmanager) (
                    1 - (increase(prometheus_notifications_successfully_sent_total{namespace="d8-monitoring"}[__SCRAPE_INTERVAL_X_4__]) / increase(prometheus_notifications_sent_total{namespace="d8-monitoring"}[__SCRAPE_INTERVAL_X_4__])) == 1),
                  "pod_ip", "$1", "alertmanager", ".*://(.*):.*")
            )
            * on (pod_ip) group_right()
            max by (madison_backend, pod, pod_ip) (
              max by (pod, pod_ip) (kube_pod_info)
              * on (pod) group_left(madison_backend)
              label_replace(
                kube_pod_labels{namespace="d8-monitoring", label_madison_backend!=""},
              "madison_backend", "$1", "label_madison_backend", "(.*)")
            )
          )
          * on (pod) group_left(deployment)
          (
            label_replace(kube_controller_pod{namespace="d8-monitoring"}, "deployment", "$1", "controller_name", "(.*)")
          )
        ) == count(increase(prometheus_notifications_sent_total{namespace="d8-monitoring", service="prometheus"}[__SCRAPE_INTERVAL_X_4__]) > 0)
      labels:
        severity_level: "5"
        tier: cluster
        d8_module: flant-integration
        d8_component: madison-proxy
      annotations:
        description: Prometheus is unable to deliver 100% alerts.
        summary: Prometheus is unable to deliver 100% alerts.
        plk_protocol_version: "1"
        plk_labels_as_annotations: "pod"
        plk_create_group_if_not_exists__d8_prometheus_madison_integration_malfunctioning: "D8PrometheusMadisonIntegrationMalfunctioning,tier=cluster,prometheus=deckhouse,namespace=d8-monitoring,deployment={{ $labels.deployment }},kubernetes=~kubernetes"
        plk_grouped_by__d8_prometheus_madison_integration_malfunctioning: "D8PrometheusMadisonIntegrationMalfunctioning,tier=cluster,prometheus=deckhouse,namespace=d8-monitoring,deployment={{ $labels.deployment }},kubernetes=~kubernetes"
